From f543db71f0cec25fa5a0ed8ecdb014d3c1dc3f66 Mon Sep 17 00:00:00 2001 From: Dan Zheng Date: Sat, 13 Oct 2018 11:04:17 -0400 Subject: [PATCH 1/2] Revamp tensor constructors. Make constructors that take both "shape" and "data" arguments: - Specify shape as an explicit `Seq[Int]`, not `Int*`. This clearly distinguishes shape from data. - Specify shape as the first argument. This is consistent with libraries like NumPy (e.g. `np.full(shape, fill_value)`). Rename `zeros(Tensor)` and `ones(Tensor)` to `zeros_like` and `ones_like`. --- src/main/scala/lantern/NNModule.scala | 16 ++--- .../scala/lantern/TensorDifferentiation.scala | 42 +++++------ src/test/scala/lantern/MnistCNNTest.scala | 18 ++--- src/test/scala/lantern/TestCublas.scala | 4 +- .../lantern/TestTensorDifferentiation.scala | 70 +++++++++---------- 5 files changed, 70 insertions(+), 80 deletions(-) diff --git a/src/main/scala/lantern/NNModule.scala b/src/main/scala/lantern/NNModule.scala index b2daeb9a..7ec32a1b 100644 --- a/src/main/scala/lantern/NNModule.scala +++ b/src/main/scala/lantern/NNModule.scala @@ -26,7 +26,7 @@ trait NNModule extends TensorDsl { for ((_, module) <- modules) module.forEachNamedParameter(f) } def enrichParameter(): Unit = { - for ((k, (tensorR, _)) <- parameters) parameters(k) = (tensorR, Some(Tensor.zeros(tensorR.x))) + for ((k, (tensorR, _)) <- parameters) parameters(k) = (tensorR, Some(Tensor.zeros_like(tensorR.x))) for ((_, module) <- modules) module.enrichParameter() } def forEachParameter(f: TensorR => Unit) = forEachNamedParameter{case (_, (tensorR, _)) => f(tensorR)} @@ -51,14 +51,14 @@ trait NNModule extends TensorDsl { case class Linear1D(val inSize: Int, val outSize: Int, val name: String = "linear1d") extends Module { val scale: Float = 1.0f / sqrt(inSize).toFloat - val weight = TensorR(Tensor.rand(scale, inSize, outSize)) + val weight = TensorR(Tensor.rand(Seq(inSize, outSize), scale)) val bias = TensorR(Tensor.zeros(outSize)) def apply(in: TensorR): TensorR @diff = in.dot(weight) + bias } case class Linear1DTrans(val inSize: Int, val outSize: Int, val name: String = "linear1dtrans") extends Module { val scale: Float = 1.0f / sqrt(inSize).toFloat - val weight = TensorR(Tensor.rand(scale, outSize, inSize)) + val weight = TensorR(Tensor.rand(Seq(outSize, inSize), scale)) val bias = TensorR(Tensor.zeros(outSize)) def apply(in: TensorR): TensorR @diff = in.dot_trans(weight) + bias } @@ -66,8 +66,8 @@ trait NNModule extends TensorDsl { case class Linear1D2(val inSize1: Int, val inSize2: Int, val outSize: Int, val name: String = "Linear1d2") extends Module { val scale1: Float = 1.0f / sqrt(inSize1).toFloat val scale2: Float = 1.0f / sqrt(inSize2).toFloat - val weight1 = TensorR(Tensor.rand(scale1, inSize1, outSize)) - val weight2 = TensorR(Tensor.rand(scale2, inSize2, outSize)) + val weight1 = TensorR(Tensor.rand(Seq(inSize1, outSize), scale1)) + val weight2 = TensorR(Tensor.rand(Seq(inSize2, outSize), scale2)) val bias = TensorR(Tensor.zeros(outSize)) def apply(in1: TensorR, in2: TensorR): TensorR @diff = in1.dot(weight1) + in2.dot(weight2) + bias } @@ -75,8 +75,8 @@ trait NNModule extends TensorDsl { case class Linear1D2Trans(val inSize1: Int, val inSize2: Int, val outSize: Int, val name: String = "Linear1d2trans") extends Module { val scale1: Float = 1.0f / sqrt(inSize1).toFloat val scale2: Float = 1.0f / sqrt(inSize2).toFloat - val weight1 = TensorR(Tensor.rand(scale1, outSize, inSize1)) - val weight2 = TensorR(Tensor.rand(scale2, outSize, inSize2)) + val weight1 = TensorR(Tensor.rand(Seq(outSize, inSize1), scale1)) + val weight2 = TensorR(Tensor.rand(Seq(outSize, inSize2), scale2)) val bias = TensorR(Tensor.zeros(outSize)) def apply(in1: TensorR, in2: TensorR): TensorR @diff = in1.dot_trans(weight1) + in2.dot_trans(weight2) + bias } @@ -85,7 +85,7 @@ trait NNModule extends TensorDsl { assert(kernelSize.size == 2, "kernel_size should be Seq[Int] of size 2") assert(stride.size == 2, "stride should be Seq[Int] of size 2") val scale: Float = 1.0f / sqrt(inChannel * kernelSize.head * kernelSize.last).toFloat - val kernel = TensorR(Tensor.rand(scale, outChannel, inChannel, kernelSize.head, kernelSize.last)) + val kernel = TensorR(Tensor.rand(Seq(outChannel, inChannel, kernelSize.head, kernelSize.last), scale)) val bias = if (useBiase) Some(TensorR(Tensor.zeros(outChannel))) else None def apply(in: TensorR): TensorR @diff = in.convBBP(kernel, bias, stride, Seq(pad, pad, pad, pad)) } diff --git a/src/main/scala/lantern/TensorDifferentiation.scala b/src/main/scala/lantern/TensorDifferentiation.scala index ae8687be..67e1ea28 100644 --- a/src/main/scala/lantern/TensorDifferentiation.scala +++ b/src/main/scala/lantern/TensorDifferentiation.scala @@ -60,7 +60,7 @@ trait TensorDsl extends DslOps with Diff { } object Dataset { - class DataLoader(name: String, train: Boolean, mean: Float, std: Float, dims: Int*) { + class DataLoader(name: String, train: Boolean, mean: Float, std: Float, dims: Seq[Int]) { val fd = open(s"../data/bin/${name}_${if (train) "train" else "test"}.bin") val len = filelen(fd) @@ -946,7 +946,7 @@ trait TensorDsl extends DslOps with Diff { val resWidth = convSize(this.shape(2) + padLeft + padRight, kernel.shape(2), strideRow) val resHeight = convSize(this.shape(3) + padUp + padDown, kernel.shape(3), strideCol) val res = bias match { - case Some(bias) => Tensor.fillWithBias(bias, 1, this.shape(0), kernel.shape(0), resWidth, resHeight) + case Some(bias) => Tensor.fillWithBias(Seq(this.shape(0), kernel.shape(0), resWidth, resHeight), bias, 1) case None => Tensor.zeros(this.shape(0), kernel.shape(0), resWidth, resHeight) } @@ -1012,7 +1012,7 @@ trait TensorDsl extends DslOps with Diff { val resWidth = convSize(this.shape(1) + padLeft + padRight, kernel.shape(2), strideRow) val resHeight = convSize(this.shape(2) + padUp + padDown, kernel.shape(3), strideCol) - val res = Tensor.fillWithBias(bias, 0, kernel.shape(0), resWidth, resHeight) + val res = Tensor.fillWithBias(Seq(kernel.shape(0), resWidth, resHeight), bias, 0) val offOut = var_new(0) // offset for the res by channel val offWeight1 = var_new(0) // offset for the kernel by channel (dim_0) @@ -1166,7 +1166,7 @@ trait TensorDsl extends DslOps with Diff { val resWidth = convSize(this.shape(2) + padUp + padDown, kernelRow, strideRow) val resHeight = convSize(this.shape(3) + padLeft + padRight, kernelCol, strideCol) - val res = Tensor.fill(0.0f, this.shape(0), this.shape(1), resWidth, resHeight) + val res = Tensor.zeros(this.shape(0), this.shape(1), resWidth, resHeight) for (i <- DataLoop(this.shape(0))) { val ptrInput = slice(this.data, i * this.shape.strides(0)) @@ -1212,7 +1212,7 @@ trait TensorDsl extends DslOps with Diff { val resHeight = this.shape(1) / strideRow val resWidth = this.shape(2) / strideCol - val res = Tensor.fill(scala.Float.MinValue, this.shape(0), resHeight, resWidth) + val res = Tensor.fill(Seq(this.shape(0), resHeight, resWidth), scala.Float.MinValue) // FIXME adhoc transform tensor to be using generic type! val savedIdx = NewArray[Int](res.scalarCount) @@ -1264,7 +1264,7 @@ trait TensorDsl extends DslOps with Diff { val resWidth = convSize(this.shape(2) + padUp + padDown, kernelRow, strideRow) val resHeight = convSize(this.shape(3) + padLeft + padRight, kernelCol, strideCol) - val res = Tensor.fill(scala.Float.MinValue, this.shape(0), this.shape(1), resWidth, resHeight) + val res = Tensor.fill(Seq(this.shape(0), this.shape(1), resWidth, resHeight), scala.Float.MinValue) val savedIdx = NewArray[Int](res.scalarCount) for (i <- DataLoop(this.shape(0))) { @@ -1366,7 +1366,7 @@ trait TensorDsl extends DslOps with Diff { val resWidth = convSize(this.shape(1) + padUp + padDown, kernelRow, strideRow) val resHeight = convSize(this.shape(2) + padLeft + padRight, kernelCol, strideCol) - val res = Tensor.fill(scala.Float.MinValue, this.shape(0), resWidth, resHeight) + val res = Tensor.fill(Seq(this.shape(0), resWidth, resHeight), scala.Float.MinValue) val savedIdx = NewArray[Int](res.scalarCount) this.maxPool_k_inplace(kernelRow, kernelCol, strideRow, strideCol, padUp, padDown, padLeft, padRight, res, savedIdx, 0) @@ -1553,7 +1553,7 @@ trait TensorDsl extends DslOps with Diff { def randseed(seed: Int) = unchecked[Unit]("srand(", seed, ")") def randseed() = unchecked[Unit]("srand(time(NULL))") def rand(dims: Int*) = randinit(dims.toSeq, 1.0f, None) - def rand(scale: Float, dims: Int*) = randinit(dims.toSeq, scale, None) + def rand(dims: Seq[Int], scale: Float) = randinit(dims.toSeq, scale, None) def randinit(dim0: Int): Tensor = randinit(Seq(dim0), 1.0f, None) def randinit(dim0: Int, seed: Option[Int]): Tensor = randinit(Seq(dim0), 1.0f, seed) def randinit(dim0: Int, dim1: Int, scale: Float): Tensor = randinit(Seq(dim0, dim1), scale, None) @@ -1577,9 +1577,9 @@ trait TensorDsl extends DslOps with Diff { new Tensor(res, dims) } - def fill(value: Rep[Float], dims: Int*): Tensor = backend.makeRepeatingTensor(dims, value) + def fill(dims: Seq[Int], value: Rep[Float]): Tensor = backend.makeRepeatingTensor(dims, value) - def fill(fFill: Seq[Rep[Int]] => Rep[Float], dims: Int*) = { + def fill(dims: Seq[Int], fFill: Seq[Rep[Int]] => Rep[Float]) = { val scalarCount = dims.product val res = backend.mallocArray[Float](scalarCount) @@ -1601,7 +1601,7 @@ trait TensorDsl extends DslOps with Diff { new Tensor(res, dims) } - def fillWithBias(bias: Tensor, dim: Int, dims: Int*) = { + def fillWithBias(dims: Seq[Int], bias: Tensor, dim: Int) = { assert(dim < dims.size && dim >= 0, s"target dimension ${dim} is out of range ${dims}") assert(bias.rank == 1 && bias.scalarCount == dims.drop(dim).head, s"bias should be 1D and have the same length as given dim") val scalarCount = dims.product @@ -1622,27 +1622,17 @@ trait TensorDsl extends DslOps with Diff { new Tensor(res, dims) } - def zeros(dims: Int*): Tensor = { - fill(0.0f, dims: _*) - } - - def zeros(that: Tensor): Tensor = { - zeros(that.shape : _*) - } - - def zeros_like(that: Tensor) = { - zeros(that.shape : _*) - } - def scalar(value: Rep[Float]) = { val res = backend.mallocArray[Float](1) res(0) = value Tensor(res, 1) } - def ones(dims: Int*) = fill(1.0f, dims: _*) - def ones(that: Tensor) = fill(1.0f, that.shape: _*) - def halves(dims: Int*) = fill(0.5f, dims: _*) + def zeros(dims: Int*): Tensor = fill(dims, 0.0f) + def zeros_like(that: Tensor) = zeros(that.shape: _*) + def ones(dims: Int*) = fill(dims, 1.0f) + def ones_like(that: Tensor) = ones(that.shape: _*) + def halves(dims: Int*) = fill(dims, 0.5f) def expand(vector: Tensor, dim1: Int) = { assert(vector.rank == 1) diff --git a/src/test/scala/lantern/MnistCNNTest.scala b/src/test/scala/lantern/MnistCNNTest.scala index c4f48b5b..3a3dfd9f 100644 --- a/src/test/scala/lantern/MnistCNNTest.scala +++ b/src/test/scala/lantern/MnistCNNTest.scala @@ -50,7 +50,7 @@ class MnistCNN extends FunSuite { val (smRow1, smCol1) = (2, 2) // FIXME scale based on PyTorch - val varConv1 = TensorR(Tensor.rand(1.0f / sqrt(inChan1 * kRow1 * kCol1).toFloat, outChan1, inChan1, kRow1, kCol1)) + val varConv1 = TensorR(Tensor.rand(Seq(outChan1, inChan1, kRow1, kCol1), 1.0f / sqrt(inChan1 * kRow1 * kCol1).toFloat)) variables += varConv1 // input size @@ -62,23 +62,23 @@ class MnistCNN extends FunSuite { // stride maxpool val (smRow2, smCol2) = (2, 2) - val varConv2 = TensorR(Tensor.rand(1.0f / sqrt(inChan2 * kRow2 * kCol2).toFloat, outChan2, inChan2, kRow2, kCol2)) + val varConv2 = TensorR(Tensor.rand(Seq(outChan2, inChan2, kRow2, kCol2), 1.0f / sqrt(inChan2 * kRow2 * kCol2).toFloat)) variables += varConv2 // Layer 3 val (oRow2, oCol2) = (convSize(iRow2, kRow2, sRow2)/smRow2, convSize(iCol2, kCol2, sCol2)/smCol2) val (in3, out3) = (outChan2 * oRow2 * oCol2, 50) // 320 - val varA1 = TensorR(Tensor.rand(1.0f / sqrt(in3).toFloat, out3, in3)) - val varB1 = TensorR(Tensor.rand(1.0f / sqrt(in3).toFloat, out3)) + val varA1 = TensorR(Tensor.rand(Seq(out3, in3), 1.0f / sqrt(in3).toFloat)) + val varB1 = TensorR(Tensor.rand(Seq(out3), 1.0f / sqrt(in3).toFloat)) variables += varA1 variables += varB1 // Layer 4 val (in4, out4) = (out3, 10) - val varA2 = TensorR(Tensor.rand(1.0f / sqrt(in4).toFloat, out4, in4)) - val varB2 = TensorR(Tensor.rand(1.0f / sqrt(in4).toFloat, out4)) + val varA2 = TensorR(Tensor.rand(Seq(out4, in4), 1.0f / sqrt(in4).toFloat)) + val varB2 = TensorR(Tensor.rand(Seq(out4), 1.0f / sqrt(in4).toFloat)) variables += varA2 variables += varB2 @@ -87,12 +87,12 @@ class MnistCNN extends FunSuite { val lr = 0.0005f val mom = 0.0f - val momentum = if (mom > 0.0f) variables map(tR => Tensor.zeros(tR.d)) else ArrayBuffer[Tensor]() + val momentum = if (mom > 0.0f) variables map(tR => Tensor.zeros_like(tR.d)) else ArrayBuffer[Tensor]() val tot1 = NewArray[Long](2) val tot2 = NewArray[Long](2) - val train = new Dataset.DataLoader("mnist", true, mean, std, iChan1, iRow1, iCol1) + val train = new Dataset.DataLoader("mnist", true, mean, std, Seq(iChan1, iRow1, iCol1)) printf("Start normalize\\n") train.normalize() @@ -240,7 +240,7 @@ class MnistCNN extends FunSuite { val tot1 = NewArray[Long](2) val tot2 = NewArray[Long](2) - val train = new Dataset.DataLoader("mnist", true, mean = 0.1307f, std = 0.3081f, iChan1, iRow1, iCol1) + val train = new Dataset.DataLoader("mnist", true, mean = 0.1307f, std = 0.3081f, Seq(iChan1, iRow1, iCol1)) train.normalize() val prepareTime = dataTimer.getElapsedTime / 1e6f diff --git a/src/test/scala/lantern/TestCublas.scala b/src/test/scala/lantern/TestCublas.scala index e78b697e..88a9060d 100644 --- a/src/test/scala/lantern/TestCublas.scala +++ b/src/test/scala/lantern/TestCublas.scala @@ -37,7 +37,7 @@ class TestCublas extends LanternFunSuite { val result = m.dot(v).toCPU() backend = BackendCPU() - val expected = Tensor.fill(4, 2) + val expected = Tensor.fill(Seq(2), 4) Tensor.assertEqual(result, expected) } } @@ -62,7 +62,7 @@ class TestCublas extends LanternFunSuite { val result = m1.dot(m2).toCPU() backend = BackendCPU() - val expected = Tensor.fill(4, 4, 4) + val expected = Tensor.fill(Seq(4, 4), 4) Tensor.assertEqual(result, expected) } } diff --git a/src/test/scala/lantern/TestTensorDifferentiation.scala b/src/test/scala/lantern/TestTensorDifferentiation.scala index 9d32061b..90f6cbde 100644 --- a/src/test/scala/lantern/TestTensorDifferentiation.scala +++ b/src/test/scala/lantern/TestTensorDifferentiation.scala @@ -151,7 +151,7 @@ class AdLMSVectorTest extends LanternFunSuite { val loss = gradR_loss(model)(Tensor.zeros(1)) Tensor.assertEqual(loss, ((vector dot vector) * Tensor.halves(dim0)).sum(), "1") Tensor.assertEqual(ve.d, vector * 2.0f ,"2") - Tensor.assertEqual(half.d, Tensor.fill((vector dot vector).data(0), 2), "3") + Tensor.assertEqual(half.d, Tensor.fill(Seq(2), (vector dot vector).data(0)), "3") () } } @@ -194,7 +194,7 @@ class AdLMSVectorTest extends LanternFunSuite { @virtualize def snippet(a: Rep[String]): Rep[Unit] = { val idx = var_new(0) - val t = Tensor.fill(seq => { idx += 1; idx }, 2, 3) + val t = Tensor.fill(Seq(2, 3), (seq => { idx += 1; idx })) Tensor.assertEqual(t.trans(), Tensor.fromData(1.0f, 4.0f, 2.0f, 5.0f, 3.0f, 6.0f).resize(3, 2), "Transpose invalid") } @@ -810,7 +810,7 @@ class AdLMSVectorTest extends LanternFunSuite { val kernel = Tensor.ones(kOut, kIn, kRow, kCol) val res = input.conv2D(kernel, 1, 1) - Tensor.assertEqual(res, Tensor.fill((kRow * kCol * kIn) * 1.0f, kOut, iRow - kRow + 1, iCol - kCol + 1), "CNN 1") + Tensor.assertEqual(res, Tensor.fill(Seq(kOut, iRow - kRow + 1, iCol - kCol + 1), (kRow * kCol * kIn) * 1.0f), "CNN 1") } } cnn_test1.eval("abc") @@ -830,10 +830,10 @@ class AdLMSVectorTest extends LanternFunSuite { val kIn = iPane val kRow = 3 val kCol = 3 - val kernel = Tensor.fill((i: Seq[Rep[Int]]) => if (i(2) == kRow/2 && i(3) == kCol/2) 1.0f else 0.0f, kOut, kIn, kRow, kCol) + val kernel = Tensor.fill(Seq(kOut, kIn, kRow, kCol), (i: Seq[Rep[Int]]) => if (i(2) == kRow/2 && i(3) == kCol/2) 1.0f else 0.0f) val res = input.conv2D(kernel, 1, 1) - Tensor.assertEqual(res, Tensor.fill(1.0f, kOut, iRow - kRow + 1, iCol - kCol + 1), "CNN 2") + Tensor.assertEqual(res, Tensor.fill(Seq(kOut, iRow - kRow + 1, iCol - kCol + 1), 1.0f), "CNN 2") } } @@ -855,10 +855,10 @@ class AdLMSVectorTest extends LanternFunSuite { val kIn = iPane val kRow = 3 val kCol = 3 - val kernel = Tensor.fill((i: Seq[Rep[Int]]) => if (i(2) == kRow/2 && i(3) == kCol/2) 1.0f else 0.0f ,kOut, kIn, kRow, kCol) + val kernel = Tensor.fill(Seq(kOut, kIn, kRow, kCol), ((i: Seq[Rep[Int]]) => if (i(2) == kRow/2 && i(3) == kCol/2) 1.0f else 0.0f)) val res = input.conv2D(kernel, 2, 2) - Tensor.assertEqual(res, Tensor.fill(1.0f, kOut, (iRow - kRow)/2 + 1, (iCol - kCol)/2 + 1), "CNN 3") + Tensor.assertEqual(res, Tensor.fill(Seq(kOut, (iRow - kRow)/2 + 1, (iCol - kCol)/2 + 1), 1.0f), "CNN 3") } } @@ -900,7 +900,7 @@ class AdLMSVectorTest extends LanternFunSuite { Tensor.assertEqual(loss, Tensor.scalar(resR * resC * 9.0f), "BACK - LOSS") - Tensor.assertEqual(varKernel.d, Tensor.fill(resR * resC * 1.0f, kIn, kOut, kRow, kCol), "BACK 1 - KERNEL D") + Tensor.assertEqual(varKernel.d, Tensor.fill(Seq(kIn, kOut, kRow, kCol), resR * resC * 1.0f), "BACK 1 - KERNEL D") } } @@ -921,7 +921,7 @@ class AdLMSVectorTest extends LanternFunSuite { val kIn = iPane val kRow = 3 val kCol = 3 - val kernel = Tensor.fill((i: Seq[Rep[Int]]) => if (i(2) == kRow/2 && i(3) == kCol/2) 1.0f else 0.0f ,kOut, kIn, kRow, kCol) + val kernel = Tensor.fill(Seq(kOut, kIn, kRow, kCol), (i: Seq[Rep[Int]]) => if (i(2) == kRow/2 && i(3) == kCol/2) 1.0f else 0.0f) val varInput = TensorR(input) val varKernel = TensorR(kernel) @@ -942,7 +942,7 @@ class AdLMSVectorTest extends LanternFunSuite { val resC = (iCol - kCol)/cS + 1 Tensor.assertEqual(loss, Tensor.scalar(resR * resC * 1.0f), "BACK 2 - LOSS") - Tensor.assertEqual(varKernel.d, Tensor.fill(resR * resC * 1.0f, kIn, kOut, kRow, kCol), "BACK 2 - KERNEL D") + Tensor.assertEqual(varKernel.d, Tensor.fill(Seq(kIn, kOut, kRow, kCol), resR * resC * 1.0f), "BACK 2 - KERNEL D") } } @@ -963,7 +963,7 @@ class AdLMSVectorTest extends LanternFunSuite { val kIn = iPane val kRow = 3 val kCol = 3 - val kernel = Tensor.fill((i: Seq[Rep[Int]]) => if (i(2) == kRow/2 && i(3) == kCol/2) 1.0f else 0.0f, kOut, kIn, kRow, kCol) + val kernel = Tensor.fill(Seq(kOut, kIn, kRow, kCol), (i: Seq[Rep[Int]]) => if (i(2) == kRow/2 && i(3) == kCol/2) 1.0f else 0.0f) val varInput = TensorR(input) val varKernel = TensorR(kernel) @@ -983,7 +983,7 @@ class AdLMSVectorTest extends LanternFunSuite { val resC = (iCol - kCol)/cS + 1 Tensor.assertEqual(loss, Tensor.scalar(resR * resC * 1.0f), "BACK 2 - LOSS") - Tensor.assertEqual(varKernel.d, Tensor.fill(resR * resC * 1.0f, kIn, kOut, kRow, kCol), "BACK 2 - KERNEL D") + Tensor.assertEqual(varKernel.d, Tensor.fill(Seq(kIn, kOut, kRow, kCol), resR * resC * 1.0f), "BACK 2 - KERNEL D") } } @@ -1009,7 +1009,7 @@ class AdLMSVectorTest extends LanternFunSuite { val rS = 2 val cS = 2 val res = input.conv2D(kernel, rS, cS) - Tensor.assertEqual(res, Tensor.fill(iPane * kRow * kCol * 1.0f, kOut, (iRow - kRow)/rS + 1, (iCol - kCol)/cS + 1), "CNN 4") + Tensor.assertEqual(res, Tensor.fill(Seq(kOut, (iRow - kRow)/rS + 1, (iCol - kCol)/cS + 1), iPane * kRow * kCol * 1.0f), "CNN 4") } } @@ -1050,7 +1050,7 @@ class AdLMSVectorTest extends LanternFunSuite { val resC = (iCol - kCol)/cS + 1 Tensor.assertEqual(loss, Tensor.scalar(kOut * resR * resC * 27.0f), "BACK 4 - LOSS") - Tensor.assertEqual(varKernel.d, Tensor.fill(resR * resC * 1.0f, kOut, kIn, kRow, kCol), "BACK 4 - KERNEL D") + Tensor.assertEqual(varKernel.d, Tensor.fill(Seq(kOut, kIn, kRow, kCol), resR * resC * 1.0f), "BACK 4 - KERNEL D") } } @@ -1077,7 +1077,7 @@ class AdLMSVectorTest extends LanternFunSuite { val rS = 2 val cS = 2 val res = input.conv2D(kernel, rS, cS) - Tensor.assertEqual(res, Tensor.fill(iPane * kRow * kCol * 1.0f, kOut, (iRow - kRow)/rS + 1, (iCol - kCol)/cS + 1), "CNN 5") + Tensor.assertEqual(res, Tensor.fill(Seq(kOut, (iRow - kRow)/rS + 1, (iCol - kCol)/cS + 1), iPane * kRow * kCol * 1.0f), "CNN 5") } } @@ -1098,7 +1098,7 @@ class AdLMSVectorTest extends LanternFunSuite { val kIn = iPane val kRow = 3 val kCol = 3 - val kernel = Tensor.fill((i: Seq[Rep[Int]]) => if (i(2) == kRow/2 && i(3) == kCol/2) 1.0f else 0.0f ,kOut, kIn, kRow, kCol) + val kernel = Tensor.fill(Seq(kOut, kIn, kRow, kCol), (i: Seq[Rep[Int]]) => if (i(2) == kRow/2 && i(3) == kCol/2) 1.0f else 0.0f) val varInput = TensorR(input) val varKernel = TensorR(kernel) @@ -1118,7 +1118,7 @@ class AdLMSVectorTest extends LanternFunSuite { val resC = (iCol - kCol)/cS + 1 Tensor.assertEqual(loss, Tensor.scalar(kOut * resR * resC * kIn * 1.0f), "BACK 5 - LOSS") - Tensor.assertEqual(varKernel.d, Tensor.fill(resR * resC * 1.0f, kOut, kIn, kRow, kCol), "BACK 5 - KERNEL D") + Tensor.assertEqual(varKernel.d, Tensor.fill(Seq(kOut, kIn, kRow, kCol), resR * resC * 1.0f), "BACK 5 - KERNEL D") } } @@ -1173,7 +1173,7 @@ class AdLMSVectorTest extends LanternFunSuite { val loss = gradR_loss(lossFun)(Tensor.zeros(1)) Tensor.assertEqual(loss, Tensor.scalar(iPane * (iRow/sR) * (iCol/sC) * 1.0f), "MAXPOOL BACK 1 - LOSS") - Tensor.assertEqual(varInput.d, Tensor.fill((i: Seq[Rep[Int]]) => if (i(1) % sR == 0 && i(2) % sC == 0) 1.0f else 0.0f, iPane, iRow, iCol), "MAXPOOL BACK 1 - D") + Tensor.assertEqual(varInput.d, Tensor.fill(Seq(iPane, iRow, iCol), (i: Seq[Rep[Int]]) => if (i(1) % sR == 0 && i(2) % sC == 0) 1.0f else 0.0f), "MAXPOOL BACK 1 - D") } @@ -1197,7 +1197,7 @@ class AdLMSVectorTest extends LanternFunSuite { // val (resNone, idxNone) = input.dropout(1.0f) Tensor.assertEqual(resAll, input, "DROPOUT 1") - // Tensor.assertEqual(resNone, Tensor.zeros(input), "DROPOUT 2") + // Tensor.assertEqual(resNone, Tensor.zeros_like(input), "DROPOUT 2") for (i <- 0 until input.scalarCount: Rep[Range]) { assertC(idxAll.data(i) == 1.0f, "idxAll incorrect %.3f != 1\\n", idxAll.data(i)) @@ -1227,7 +1227,7 @@ class AdLMSVectorTest extends LanternFunSuite { } val loss = gradR_loss(lossFun)(Tensor.zeros(1)) - Tensor.assertEqual(varInput.d, Tensor.ones(input), "DROPOUT BACK 1 - D") + Tensor.assertEqual(varInput.d, Tensor.ones_like(input), "DROPOUT BACK 1 - D") } } @@ -1253,7 +1253,7 @@ class AdLMSVectorTest extends LanternFunSuite { // } // val loss = gradR_loss(lossFun)(Tensor.zeros(1)) - // Tensor.assertEqual(varInput.d, Tensor.zeros(input), "DROPOUT BACK 1 - D") + // Tensor.assertEqual(varInput.d, Tensor.zeros_like(input), "DROPOUT BACK 1 - D") // } // } @@ -1273,7 +1273,7 @@ class AdLMSVectorTest extends LanternFunSuite { val iRow1 = 10 val iCol1 = 10 - val input = Tensor.rand(1.0f, iChan1, iRow1, iCol1) + val input = Tensor.rand(Seq(iChan1, iRow1, iCol1), 1.0f) // Layer 1 val inChan1 = iChan1 @@ -1289,22 +1289,22 @@ class AdLMSVectorTest extends LanternFunSuite { val smRow1 = 2 val smCol1 = 2 - val conv1 = Tensor.rand(1.0f, outChan1, inChan1, kRow1, kCol1) + val conv1 = Tensor.rand(Seq(outChan1, inChan1, kRow1, kCol1), 1.0f) val oRow1 = convSize(iRow1, kRow1, sRow1)/smRow1 val oCol1 = convSize(iCol1, kCol1, sCol1)/smCol1 val inChan2 = outChan1 val outChan2 = 3 - val conv2 = Tensor.rand(1.0f, outChan2, inChan2, kRow1, kCol1) + val conv2 = Tensor.rand(Seq(outChan2, inChan2, kRow1, kCol1), 1.0f) val oRow2 = convSize(oRow1, kRow1, sRow1) val oCol2 = convSize(oCol1, kCol1, sCol1) val out3 = 5 val in3 = outChan2 * oRow2 * oCol2 - val a1 = Tensor.rand(1.0f, out3, in3) - val b1 = Tensor.rand(1.0f, out3) + val a1 = Tensor.rand(Seq(out3, in3), 1.0f) + val b1 = Tensor.rand(Seq(out3), 1.0f) val varInput = TensorR(input) @@ -1441,7 +1441,7 @@ class AdLMSVectorTest extends LanternFunSuite { // assert equal val expect_input_grad = Tensor.fromData(scala.collection.Seq(1,1,4,4), 1.0f, 2.0f, 2.0f, 1.0f, 2.0f, 4.0f, 4.0f, 2.0f, 2.0f, 4.0f, 4.0f, 2.0f, 1.0f, 2.0f, 2.0f, 1.0f) - val expect_kernel_grad = Tensor.fill(4.0f, 1, 1, 3, 3) + val expect_kernel_grad = Tensor.fill(Seq(1, 1, 3, 3), 4.0f) val expect_bias_grad = Tensor.fromData(scala.collection.Seq(1), 4.0f) Tensor.assertEqual(expect_input_grad * 2.0f, input.d, "expect and input.gradient are") Tensor.assertEqual(expect_kernel_grad * 2.0f, kernel.d, "expect and kernel.gradient are") @@ -1496,7 +1496,7 @@ class AdLMSVectorTest extends LanternFunSuite { } gradR_loss(lossFun)(Tensor.zeros(1)) // assert equal - val expected_grad = Tensor.fill(0.25f, 1, 1, 4, 4) + val expected_grad = Tensor.fill(Seq(1, 1, 4, 4), 0.25f) Tensor.assertEqual(expected_grad, input.d, "expect and input.gradient are") input.clear_grad() @@ -1517,24 +1517,24 @@ class AdLMSVectorTest extends LanternFunSuite { @virtualize def snippet(a: Rep[String]): Rep[Unit] = { val input1 = Tensor.ones(3,4,5) - Tensor.assertEqual(input1.sum(dim = 0), Tensor.fill(3.0f, 4, 5)) - Tensor.assertEqual(input1.sum(dim = 1), Tensor.fill(4.0f, 3, 5)) - Tensor.assertEqual(input1.sum(dim = 2), Tensor.fill(5.0f, 3, 4)) + Tensor.assertEqual(input1.sum(dim = 0), Tensor.fill(Seq(4, 5), 3.0f)) + Tensor.assertEqual(input1.sum(dim = 1), Tensor.fill(Seq(3, 5), 4.0f)) + Tensor.assertEqual(input1.sum(dim = 2), Tensor.fill(Seq(3, 4), 5.0f)) val input2 = TensorR(input1) gradR_loss(dummy => input2.sum(dim = 0).sum())(Tensor.zeros(1)) - Tensor.assertEqual(input2.d, Tensor.fill(1.0f, 3, 4, 5)) + Tensor.assertEqual(input2.d, Tensor.fill(Seq(3, 4, 5), 1.0f)) gradR_loss(dummy => input2.sum(dim = 1).sum())(Tensor.zeros(1)) - Tensor.assertEqual(input2.d, Tensor.fill(2.0f, 3, 4, 5)) + Tensor.assertEqual(input2.d, Tensor.fill(Seq(3, 4, 5), 2.0f)) gradR_loss(dummy => input2.sum(dim = 2).sum())(Tensor.zeros(1)) - Tensor.assertEqual(input2.d, Tensor.fill(3.0f, 3, 4, 5)) + Tensor.assertEqual(input2.d, Tensor.fill(Seq(3, 4, 5), 3.0f)) val input3 = Tensor.ones(2,4,5,5) Tensor.assertEqual(input3.sum(3).sum(2).sum(0).resize(-1, 1, 1), input3.batchNormAv() * 2 * 5 * 5) val input4 = TensorR(input3) gradR_loss(dummy => input4.batchNormAv().sum())(Tensor.zeros(1)) - Tensor.assertEqual(input4.d, Tensor.fill(1.0f / 50, 2, 4, 5, 5)) + Tensor.assertEqual(input4.d, Tensor.fill(Seq(2, 4, 5, 5), 1.0f / 50)) val input5 = TensorR(new Tensor(Array(2.0f, 3.0f), Seq(2))) val input6 = TensorR(new Tensor(Array(2.0f, 3.0f), Seq(2))) From 367cb67e9d50b56fe3719493f9e27b9e4da5c782 Mon Sep 17 00:00:00 2001 From: Dan Zheng Date: Sat, 13 Oct 2018 11:48:52 -0400 Subject: [PATCH 2/2] Clean up `Seq` usages. --- .../lantern/TestTensorDifferentiation.scala | 43 +++++++++---------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/src/test/scala/lantern/TestTensorDifferentiation.scala b/src/test/scala/lantern/TestTensorDifferentiation.scala index 90f6cbde..92a26dda 100644 --- a/src/test/scala/lantern/TestTensorDifferentiation.scala +++ b/src/test/scala/lantern/TestTensorDifferentiation.scala @@ -1344,19 +1344,18 @@ class AdLMSVectorTest extends LanternFunSuite { test("op_conv") { val deb = new LanternDriverC[String, Unit] { - import scala.collection.Seq; @virtualize def snippet(a: Rep[String]): Rep[Unit] = { val input = Tensor.ones(1, 3, 8, 8) val kernel = Tensor.ones(1, 3, 3, 3) val bias = Tensor.ones(1) - val strides: Seq[Int] = List(2, 2).toSeq - val pads: Seq[Int] = List(0,0,0,0).toSeq + val strides = Seq(2, 2) + val pads = Seq(0,0,0,0) val output = input.conv2D_batch(kernel, Some(bias), strides, pads) // assert equal - val expect = Tensor.fromData(scala.collection.Seq(1,1,3,3), 28.0f, 28.0f, 28.0f, 28.0f, 28.0f, 28.0f, 28.0f, 28.0f, 28.0f) + val expect = Tensor.fromData(Seq(1,1,3,3), 28.0f, 28.0f, 28.0f, 28.0f, 28.0f, 28.0f, 28.0f, 28.0f, 28.0f) Tensor.assertEqual(expect, output, "expect and output are") } } @@ -1372,19 +1371,18 @@ class AdLMSVectorTest extends LanternFunSuite { test("op_conv_pad") { val deb = new LanternDriverC[String, Unit] { - import scala.collection.Seq; @virtualize def snippet(a: Rep[String]): Rep[Unit] = { val input = Tensor.ones(1, 1, 4, 4) val kernel = Tensor.ones(1, 1, 3, 3) val bias = Tensor.zeros(1) - val strides: Seq[Int] = List(3, 3).toSeq - val pads: Seq[Int] = List(1, 1, 1, 1).toSeq + val strides = Seq(3, 3) + val pads = Seq(1, 1, 1, 1) val output = input.conv2D_batch(kernel, Some(bias), strides, pads) // assert equal - val expect = Tensor.fromData(scala.collection.Seq(1,1,2,2), 4.0f, 4.0f, 4.0f, 4.0f) + val expect = Tensor.fromData(Seq(1,1,2,2), 4.0f, 4.0f, 4.0f, 4.0f) Tensor.assertEqual(expect, output, "expect and output are") } } @@ -1398,17 +1396,16 @@ class AdLMSVectorTest extends LanternFunSuite { test("op_conv_pad_nobias") { val deb = new LanternDriverC[String, Unit] { - import scala.collection.Seq; @virtualize def snippet(a: Rep[String]): Rep[Unit] = { val input = Tensor.ones(1, 1, 4, 4) val kernel = Tensor.ones(1, 1, 3, 3) - val strides: Seq[Int] = List(3, 3).toSeq - val pads: Seq[Int] = List(1, 1, 1, 1).toSeq + val strides = Seq(3, 3) + val pads = Seq(1, 1, 1, 1) val output = input.conv2D_batch(kernel, None, strides, pads) - val expect = Tensor.fromData(scala.collection.Seq(1,1,2,2), 4.0f, 4.0f, 4.0f, 4.0f) + val expect = Tensor.fromData(Seq(1,1,2,2), 4.0f, 4.0f, 4.0f, 4.0f) Tensor.assertEqual(expect, output, "expect and output are") } } @@ -1428,8 +1425,8 @@ class AdLMSVectorTest extends LanternFunSuite { val input = TensorR(Tensor.ones(1,1,4,4)) val kernel = TensorR(Tensor.ones(1,1,3,3)) val bias = TensorR(Tensor.zeros(1)) - val strides: scala.collection.Seq[Int] = List(1,1).toSeq - val pads: scala.collection.Seq[Int] = List(0,0,0,0).toSeq + val strides = Seq(1,1) + val pads = Seq(0,0,0,0) def lossFun(x: TensorR) = { val output = input.convBBP(kernel, Some(bias), strides, pads) @@ -1439,10 +1436,10 @@ class AdLMSVectorTest extends LanternFunSuite { gradR_loss(lossFun)(Tensor.zeros(1)) // assert equal - val expect_input_grad = Tensor.fromData(scala.collection.Seq(1,1,4,4), + val expect_input_grad = Tensor.fromData(Seq(1,1,4,4), 1.0f, 2.0f, 2.0f, 1.0f, 2.0f, 4.0f, 4.0f, 2.0f, 2.0f, 4.0f, 4.0f, 2.0f, 1.0f, 2.0f, 2.0f, 1.0f) val expect_kernel_grad = Tensor.fill(Seq(1, 1, 3, 3), 4.0f) - val expect_bias_grad = Tensor.fromData(scala.collection.Seq(1), 4.0f) + val expect_bias_grad = Tensor.fromData(Seq(1), 4.0f) Tensor.assertEqual(expect_input_grad * 2.0f, input.d, "expect and input.gradient are") Tensor.assertEqual(expect_kernel_grad * 2.0f, kernel.d, "expect and kernel.gradient are") Tensor.assertEqual(expect_bias_grad * 2.0f, bias.d, "expect and bias.gradient are") @@ -1459,8 +1456,8 @@ class AdLMSVectorTest extends LanternFunSuite { val input = TensorR(Tensor.ones(1,1,4,4)) val kernel = TensorR(Tensor.ones(1,1,3,3)) val bias = TensorR(Tensor.zeros(1)) - val strides: scala.collection.Seq[Int] = List(3,3).toSeq - val pads: scala.collection.Seq[Int] = List(1,1,1,1).toSeq + val strides = Seq(3,3) + val pads = Seq(1,1,1,1) def lossFun(x: TensorR) = { val output = input.convBBP(kernel, Some(bias), strides, pads) @@ -1469,11 +1466,11 @@ class AdLMSVectorTest extends LanternFunSuite { val loss = gradR_loss(lossFun)(Tensor.zeros(1)) // assert equal - val expect_input_grad = Tensor.fromData(scala.collection.Seq(1,1,4,4), + val expect_input_grad = Tensor.fromData(Seq(1,1,4,4), 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f) - val expect_kernel_grad = Tensor.fromData(scala.collection.Seq(1,1,3,3), + val expect_kernel_grad = Tensor.fromData(Seq(1,1,3,3), 1.0f, 2.0f, 1.0f, 2.0f, 4.0f, 2.0f, 1.0f, 2.0f, 1.0f) - val expect_bias_grad = Tensor.fromData(scala.collection.Seq(1), 4.0f) + val expect_bias_grad = Tensor.fromData(Seq(1), 4.0f) Tensor.assertEqual(expect_input_grad, input.d, "expect and input.gradient are") Tensor.assertEqual(expect_kernel_grad, kernel.d, "expect and kernel.gradient are") Tensor.assertEqual(expect_bias_grad, bias.d, "expect and bias.gradient are") @@ -1492,7 +1489,7 @@ class AdLMSVectorTest extends LanternFunSuite { def snippet(a: Rep[String]): Rep[Unit] = { val input = TensorR(Tensor.ones(1,1,4,4)) def lossFun(x: TensorR) = { - input.averagePoolBK(List(2, 2).toSeq, List(2, 2).toSeq, None).sum() + input.averagePoolBK(Seq(2, 2), Seq(2, 2), None).sum() } gradR_loss(lossFun)(Tensor.zeros(1)) // assert equal @@ -1501,7 +1498,7 @@ class AdLMSVectorTest extends LanternFunSuite { input.clear_grad() def lossFun2(x: TensorR) = { - input.averagePoolBK(List(2, 2).toSeq, List(1, 1).toSeq, None).sum() + input.averagePoolBK(Seq(2, 2), Seq(1, 1), None).sum() } gradR_loss(lossFun2)(Tensor.zeros(1)) // assert equal