From 0f6c6843184056e49105860696dc0eeffcc8057d Mon Sep 17 00:00:00 2001 From: Franklin Hu Date: Tue, 24 Feb 2015 09:43:57 -0800 Subject: [PATCH 1/9] Add correct String/ChannelBuffer injections --- .../com/twitter/storehaus/mysql/ValueMapper.scala | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/storehaus-mysql/src/main/scala/com/twitter/storehaus/mysql/ValueMapper.scala b/storehaus-mysql/src/main/scala/com/twitter/storehaus/mysql/ValueMapper.scala index c5f6a7af..faf54e23 100644 --- a/storehaus-mysql/src/main/scala/com/twitter/storehaus/mysql/ValueMapper.scala +++ b/storehaus-mysql/src/main/scala/com/twitter/storehaus/mysql/ValueMapper.scala @@ -124,21 +124,33 @@ class MySqlValue(val v: Value) { * Returns string representation of the finagle-mysql Value wrapped by MySqlValue * Both null values and empty values map to empty string. */ +@deprecated("Use String2MySqlValueInjection", "0.10.0") object MySqlStringInjection extends Injection[MySqlValue, String] { def apply(a: MySqlValue): String = ValueMapper.toString(a.v).getOrElse("") // should this be null: String instead? override def invert(b: String) = Try(MySqlValue(RawValue(Type.String, Charset.Utf8_general_ci, false, b.getBytes))) } +object String2MySqlValueInjection extends Injection[String, MySqlValue] { + def apply(s: String): MySqlValue = MySqlValue(s) + override def invert(m: MySqlValue): Try[String] = Try { ValueMapper.toString(m.v).get } +} + /** * Injection from MySqlValue to ChannelBuffer. * Returns a channel buffer containing the Value wrapped by MySqlValue. * Both null values and empty values map to empty channel buffer. */ +@deprecated("Use ChannelBuffer2MySqlValueInjection", "0.10.0") object MySqlCbInjection extends Injection[MySqlValue, ChannelBuffer] { def apply(a: MySqlValue): ChannelBuffer = ValueMapper.toChannelBuffer(a.v).getOrElse(ChannelBuffers.EMPTY_BUFFER) override def invert(b: ChannelBuffer) = Try(MySqlValue((Type.String, Charset.Utf8_general_ci, false, b.toString(UTF_8)))) } +object ChannelBuffer2MySqlValueInjection extends Injection[ChannelBuffer, MySqlValue] { + def apply(c: ChannelBuffer): MySqlValue = MySqlValue(c) + override def invert(m: MySqlValue): Try[ChannelBuffer] = Try { ValueMapper.toChannelBuffer(m.v).get } +} + object LongMySqlInjection extends Injection[Long, MySqlValue] { def apply(a: Long): MySqlValue = MySqlValue(a) override def invert(b: MySqlValue) = Try(ValueMapper.toLong(b.v).get) From 42ee798bb6b573977481c4bacd23adc3e4368b36 Mon Sep 17 00:00:00 2001 From: Franklin Hu Date: Fri, 27 Feb 2015 14:50:38 -0800 Subject: [PATCH 2/9] Fix bug in ValueMapper ChannelBuffer conversion, add tests --- .../twitter/storehaus/mysql/ValueMapper.scala | 7 ++- .../mysql/MySqlValueInjectionProperties.scala | 60 +++++++++++++++++++ 2 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 storehaus-mysql/src/test/scala/com/twitter/storehaus/mysql/MySqlValueInjectionProperties.scala diff --git a/storehaus-mysql/src/main/scala/com/twitter/storehaus/mysql/ValueMapper.scala b/storehaus-mysql/src/main/scala/com/twitter/storehaus/mysql/ValueMapper.scala index faf54e23..2f500989 100644 --- a/storehaus-mysql/src/main/scala/com/twitter/storehaus/mysql/ValueMapper.scala +++ b/storehaus-mysql/src/main/scala/com/twitter/storehaus/mysql/ValueMapper.scala @@ -95,7 +95,12 @@ object MySqlValue { case v: Int => new MySqlValue(IntValue(v)) case v: Long => new MySqlValue(LongValue(v)) case v: Short => new MySqlValue(ShortValue(v)) - case v: ChannelBuffer => new MySqlValue(RawValue(Type.String, Charset.Utf8_general_ci, false, v.toString(UTF_8).getBytes)) + case v: ChannelBuffer => + val bytes = Array.ofDim[Byte](v.readableBytes) + v.markReaderIndex() + v.readBytes(bytes) + v.resetReaderIndex() + new MySqlValue(RawValue(Type.Blob, Charset.Binary, isBinary = true, bytes)) case _ => throw new UnsupportedOperationException(v.getClass.getName + " is currently not supported.") } } diff --git a/storehaus-mysql/src/test/scala/com/twitter/storehaus/mysql/MySqlValueInjectionProperties.scala b/storehaus-mysql/src/test/scala/com/twitter/storehaus/mysql/MySqlValueInjectionProperties.scala new file mode 100644 index 00000000..f965f9c7 --- /dev/null +++ b/storehaus-mysql/src/test/scala/com/twitter/storehaus/mysql/MySqlValueInjectionProperties.scala @@ -0,0 +1,60 @@ +/* + * Copyright 2013 Twitter Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.twitter.storehaus.mysql + +import java.util.Arrays + +import org.jboss.netty.buffer.{ChannelBuffers, ChannelBuffer} +import org.scalacheck.{Gen, Arbitrary, Properties} +import org.scalacheck.Prop.forAll + +object MySqlValueInjectionProperties extends Properties("MySqlValue Injections") { + + val channelBufferGenerator = for { + b <- Gen.containerOf[Array, Byte](Arbitrary.arbitrary[Byte]) + } yield ChannelBuffers.wrappedBuffer(b) + + implicit val gen = Arbitrary(channelBufferGenerator) + + property("String2MySqlValue apply and invert") = { + val injection = String2MySqlValueInjection + forAll { (s: String) => + injection.invert(injection(s)).toOption == Some(s) + } + } + + property("ChannelBuffer2MySqlValue apply and invert") = { + val injection = ChannelBuffer2MySqlValueInjection + + forAll { (c: ChannelBuffer) => + val bytes = bytesFromChannelBuffer(c) + val inverted = injection.invert(injection(c)).toOption + inverted.exists { invertedBuf => + val invertedBytes = bytesFromChannelBuffer(invertedBuf) + Arrays.equals(invertedBytes, bytes) + } + } + } + + def bytesFromChannelBuffer(c: ChannelBuffer): Array[Byte] = { + val arr = Array.ofDim[Byte](c.readableBytes) + c.markReaderIndex() + c.readBytes(arr) + c.resetReaderIndex() + arr + } +} From 59c134ea618a396b5853ef0c2b3633919faf37f1 Mon Sep 17 00:00:00 2001 From: Ian O Connell Date: Tue, 5 May 2015 14:54:11 -0700 Subject: [PATCH 3/9] Remove usage of twitter's maven repo, travis seems to dislike it -- maybe routing issues? --- project/Build.scala | 14 +++++++++----- project/Finagle.scala | 11 ----------- 2 files changed, 9 insertions(+), 16 deletions(-) delete mode 100644 project/Finagle.scala diff --git a/project/Build.scala b/project/Build.scala index 981d63ce..ab05ea56 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -66,7 +66,6 @@ object StorehausBuild extends Build { resolvers ++= Seq( Opts.resolver.sonatypeSnapshots, Opts.resolver.sonatypeReleases, - "Twitter Maven" at "http://maven.twttr.com", "Conjars Repository" at "http://conjars.org/repo" ), parallelExecution in Test := true, @@ -123,6 +122,7 @@ object StorehausBuild extends Build { val bijectionVersion = "0.7.2" val utilVersion = "6.22.0" val scaldingVersion = "0.13.1" + val finagleVersion = "6.22.0" lazy val storehaus = Project( id = "storehaus", @@ -182,19 +182,23 @@ object StorehausBuild extends Build { "com.twitter" %% "algebird-core" % algebirdVersion, "com.twitter" %% "bijection-core" % bijectionVersion, "com.twitter" %% "bijection-netty" % bijectionVersion, - Finagle.module("memcached") + "com.twitter" %% "finagle-memcached" % finagleVersion excludeAll( + // we don't use this and its not on maven central. + ExclusionRule("com.twitter.common.zookeeper"), + ExclusionRule("com.twitter.common") + ) ) ).dependsOn(storehausAlgebra % "test->test;compile->compile") lazy val storehausMySQL = module("mysql").settings( - libraryDependencies += Finagle.module("mysql") + libraryDependencies += "com.twitter" %% "finagle-mysql" % finagleVersion ).dependsOn(storehausAlgebra % "test->test;compile->compile") lazy val storehausRedis = module("redis").settings( libraryDependencies ++= Seq ( "com.twitter" %% "bijection-core" % bijectionVersion, "com.twitter" %% "bijection-netty" % bijectionVersion, - Finagle.module("redis") + "com.twitter" %% "finagle-redis" % finagleVersion ), // we don't want various tests clobbering each others keys parallelExecution in Test := false @@ -295,7 +299,7 @@ object StorehausBuild extends Build { lazy val storehausHttp = module("http").settings( libraryDependencies ++= Seq( - Finagle.module("http"), + "com.twitter" %% "finagle-http" % finagleVersion, "com.twitter" %% "bijection-netty" % bijectionVersion ) ).dependsOn(storehausCore) diff --git a/project/Finagle.scala b/project/Finagle.scala deleted file mode 100644 index 753e4305..00000000 --- a/project/Finagle.scala +++ /dev/null @@ -1,11 +0,0 @@ -package storehaus - -/** Module defining latest finagle version - * and means of constructing finagle module - * dependency */ -object Finagle { - import sbt._ - val LatestVersion = "6.22.0" - def module(name: String, version: String = LatestVersion) = - StorehausBuild.withCross("com.twitter" %% "finagle-%s".format(name) % version) -} From c82f083e08af170094709df737aa3832b76f5636 Mon Sep 17 00:00:00 2001 From: Franklin Hu Date: Mon, 4 May 2015 09:55:29 -1000 Subject: [PATCH 4/9] Build.scala: move spec2 to a per-module dependency This makes it easier to migrate each module to scalatest individually. Ignored the following modules since they don't need specs2: * storehaus-algebra * storehaus-caliper * storehaus-core * storehaus-dynamodb * storehaus-hbase * storehaus-http * storehaus-memcache * storehaus-mongodb * storehaus-mysql * storehaus-testing --- project/Build.scala | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/project/Build.scala b/project/Build.scala index ab05ea56..c15e72a3 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -28,15 +28,9 @@ import AssemblyKeys._ object StorehausBuild extends Build { def withCross(dep: ModuleID) = dep cross CrossVersion.binaryMapped { - case "2.9.3" => "2.9.2" // TODO: hack because twitter hasn't built things against 2.9.3 case version if version startsWith "2.10" => "2.10" // TODO: hack because sbt is broken case x => x } - - def specs2Import(scalaVersion: String) = scalaVersion match { - case version if version startsWith "2.9" => "org.specs2" %% "specs2" % "1.12.4.1" % "test" - case version if version startsWith "2.10" => "org.specs2" %% "specs2" % "1.13" % "test" - } val extraSettings = Project.defaultSettings ++ Boilerplate.settings ++ assemblySettings ++ mimaDefaultSettings @@ -62,7 +56,6 @@ object StorehausBuild extends Build { crossScalaVersions := Seq("2.10.4"), javacOptions ++= Seq("-source", "1.6", "-target", "1.6"), javacOptions in doc := Seq("-source", "1.6"), - libraryDependencies <+= scalaVersion(specs2Import(_)), resolvers ++= Seq( Opts.resolver.sonatypeSnapshots, Opts.resolver.sonatypeReleases, @@ -123,6 +116,7 @@ object StorehausBuild extends Build { val utilVersion = "6.22.0" val scaldingVersion = "0.13.1" val finagleVersion = "6.22.0" + val specs2Version = "1.13" lazy val storehaus = Project( id = "storehaus", @@ -159,7 +153,8 @@ object StorehausBuild extends Build { lazy val storehausCache = module("cache").settings( libraryDependencies += "com.twitter" %% "algebird-core" % algebirdVersion, - libraryDependencies += withCross("com.twitter" %% "util-core" % utilVersion) + libraryDependencies += withCross("com.twitter" %% "util-core" % utilVersion), + libraryDependencies += "org.specs2" %% "specs2" % specs2Version % "test" ) lazy val storehausCore = module("core").settings( @@ -198,7 +193,8 @@ object StorehausBuild extends Build { libraryDependencies ++= Seq ( "com.twitter" %% "bijection-core" % bijectionVersion, "com.twitter" %% "bijection-netty" % bijectionVersion, - "com.twitter" %% "finagle-redis" % finagleVersion + "com.twitter" %% "finagle-redis" % finagleVersion, + "org.specs2" %% "specs2" % specs2Version % "test" ), // we don't want various tests clobbering each others keys parallelExecution in Test := false @@ -238,7 +234,8 @@ object StorehausBuild extends Build { ExclusionRule("com.sun.jdmk","jmxtools"), ExclusionRule( "com.sun.jmx","jmxri"), ExclusionRule( "javax.jms","jms") - ) + ), + "org.specs2" %% "specs2" % specs2Version % "test" ), // we don't want various tests clobbering each others keys parallelExecution in Test := false @@ -251,7 +248,8 @@ object StorehausBuild extends Build { "org.apache.kafka" % "kafka_2.9.2" % "0.8.0" % "provided" excludeAll( ExclusionRule(organization = "com.sun.jdmk"), ExclusionRule(organization = "com.sun.jmx"), - ExclusionRule(organization = "javax.jms")) + ExclusionRule(organization = "javax.jms")), + "org.specs2" %% "specs2" % specs2Version % "test" ), // we don't want various tests clobbering each others keys parallelExecution in Test := false @@ -270,7 +268,8 @@ object StorehausBuild extends Build { "org.elasticsearch" % "elasticsearch" % "0.90.9", "org.json4s" %% "json4s-native" % "3.2.6", "com.google.code.findbugs" % "jsr305" % "1.3.+", - "com.twitter" %% "bijection-json4s" % bijectionVersion + "com.twitter" %% "bijection-json4s" % bijectionVersion, + "org.specs2" %% "specs2" % specs2Version % "test" ), // we don't want various tests clobbering each others keys parallelExecution in Test := false @@ -283,8 +282,10 @@ object StorehausBuild extends Build { settings = sharedSettings ++ Seq( name := "storehaus-testing", previousArtifact := youngestForwardCompatible("testing"), - libraryDependencies ++= Seq("org.scalacheck" %% "scalacheck" % "1.10.0" withSources(), - withCross("com.twitter" %% "util-core" % utilVersion)) + libraryDependencies ++= Seq( + "org.scalacheck" %% "scalacheck" % "1.10.0" withSources(), + withCross("com.twitter" %% "util-core" % utilVersion) + ) ) ) @@ -293,8 +294,9 @@ object StorehausBuild extends Build { "com.google.code.java-allocation-instrumenter" % "java-allocation-instrumenter" % "2.0", "com.google.code.gson" % "gson" % "1.7.1", "com.twitter" %% "bijection-core" % bijectionVersion, - "com.twitter" %% "algebird-core" % algebirdVersion), - javaOptions in run <++= (fullClasspath in Runtime) map { cp => Seq("-cp", sbt.Build.data(cp).mkString(":")) } + "com.twitter" %% "algebird-core" % algebirdVersion + ), + javaOptions in run <++= (fullClasspath in Runtime) map { cp => Seq("-cp", sbt.Build.data(cp).mkString(":")) } ).dependsOn(storehausCore, storehausAlgebra, storehausCache) lazy val storehausHttp = module("http").settings( From 755c78da0a07a05de5c6f99fdd03ae77cfe1339b Mon Sep 17 00:00:00 2001 From: Franklin Hu Date: Mon, 4 May 2015 10:01:20 -1000 Subject: [PATCH 5/9] Build.scala: add scalatest as a shared test dependency --- project/Build.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/project/Build.scala b/project/Build.scala index c15e72a3..d83b8a24 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -56,6 +56,7 @@ object StorehausBuild extends Build { crossScalaVersions := Seq("2.10.4"), javacOptions ++= Seq("-source", "1.6", "-target", "1.6"), javacOptions in doc := Seq("-source", "1.6"), + libraryDependencies += "org.scalatest" %% "scalatest" % scalatestVersion % "test", resolvers ++= Seq( Opts.resolver.sonatypeSnapshots, Opts.resolver.sonatypeReleases, @@ -116,6 +117,7 @@ object StorehausBuild extends Build { val utilVersion = "6.22.0" val scaldingVersion = "0.13.1" val finagleVersion = "6.22.0" + val scalatestVersion = "2.2.4" val specs2Version = "1.13" lazy val storehaus = Project( From 0bd5666b2627e85f13e708a28d3a63db11b1c9e6 Mon Sep 17 00:00:00 2001 From: Franklin Hu Date: Mon, 4 May 2015 11:00:48 -1000 Subject: [PATCH 6/9] storehaus-cache: switch to scalatest --- project/Build.scala | 3 +-- .../storehaus/cache/HHFilteredCacheTest.scala | 6 +++--- .../com/twitter/storehaus/cache/LRUCacheTest.scala | 10 +++++----- .../storehaus/cache/MutableLRUCacheTest.scala | 6 +++--- .../storehaus/cache/MutableTTLCacheTest.scala | 12 ++++++------ .../com/twitter/storehaus/cache/TTLCacheTest.scala | 13 ++++++------- 6 files changed, 24 insertions(+), 26 deletions(-) diff --git a/project/Build.scala b/project/Build.scala index d83b8a24..c62d2bb9 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -155,8 +155,7 @@ object StorehausBuild extends Build { lazy val storehausCache = module("cache").settings( libraryDependencies += "com.twitter" %% "algebird-core" % algebirdVersion, - libraryDependencies += withCross("com.twitter" %% "util-core" % utilVersion), - libraryDependencies += "org.specs2" %% "specs2" % specs2Version % "test" + libraryDependencies += withCross("com.twitter" %% "util-core" % utilVersion) ) lazy val storehausCore = module("core").settings( diff --git a/storehaus-cache/src/test/scala/com/twitter/storehaus/cache/HHFilteredCacheTest.scala b/storehaus-cache/src/test/scala/com/twitter/storehaus/cache/HHFilteredCacheTest.scala index 08b6c3e4..a0042b03 100644 --- a/storehaus-cache/src/test/scala/com/twitter/storehaus/cache/HHFilteredCacheTest.scala +++ b/storehaus-cache/src/test/scala/com/twitter/storehaus/cache/HHFilteredCacheTest.scala @@ -16,14 +16,14 @@ package com.twitter.storehaus.cache -import org.specs2.mutable._ +import org.scalatest.{Matchers, WordSpec} -class HHFilteredCacheTest extends Specification { +class HHFilteredCacheTest extends WordSpec with Matchers { def checkCache[K, V](pairs: Seq[(K, V)], m: Map[K, V])(implicit cache: MutableCache[K, V]) = { pairs.foldLeft(cache)(_ += _) val res = cache.iterator.toMap cache.clear - res must be_==(m) + res should equal(m) } "HHFilteredCache works properly" in { diff --git a/storehaus-cache/src/test/scala/com/twitter/storehaus/cache/LRUCacheTest.scala b/storehaus-cache/src/test/scala/com/twitter/storehaus/cache/LRUCacheTest.scala index 24cd3cc3..0941c83e 100644 --- a/storehaus-cache/src/test/scala/com/twitter/storehaus/cache/LRUCacheTest.scala +++ b/storehaus-cache/src/test/scala/com/twitter/storehaus/cache/LRUCacheTest.scala @@ -16,11 +16,11 @@ package com.twitter.storehaus.cache -import org.specs2.mutable._ +import org.scalatest.{Matchers, WordSpec} -class LRUCacheTest extends Specification { +class LRUCacheTest extends WordSpec with Matchers { def checkCache[K, V](pairs: Seq[(K, V)], m: Map[K, V])(implicit cache: Cache[K, V]) = - pairs.foldLeft(cache)(_ + _).toMap must be_==(m) + pairs.foldLeft(cache)(_ + _).toMap should equal(m) "LRUCache works properly with threshold 2" in { implicit val cache = Cache.lru[String, Int](2) @@ -36,7 +36,7 @@ class LRUCacheTest extends Specification { Seq("a" -> 1, "b" -> 2, "b" -> 3), Map("a" -> 1, "b" -> 3) ) - ((cache + ("a" -> 1) + ("b" -> 2)).hit("a") + ("c" -> 3)).toMap - .must(be_==(Map("a" -> 1, "c" -> 3))) + val result = ((cache + ("a" -> 1) + ("b" -> 2)).hit("a") + ("c" -> 3)).toMap + result should equal(Map("a" -> 1, "c" -> 3)) } } diff --git a/storehaus-cache/src/test/scala/com/twitter/storehaus/cache/MutableLRUCacheTest.scala b/storehaus-cache/src/test/scala/com/twitter/storehaus/cache/MutableLRUCacheTest.scala index 9abf39fc..200b5c7b 100644 --- a/storehaus-cache/src/test/scala/com/twitter/storehaus/cache/MutableLRUCacheTest.scala +++ b/storehaus-cache/src/test/scala/com/twitter/storehaus/cache/MutableLRUCacheTest.scala @@ -16,15 +16,15 @@ package com.twitter.storehaus.cache -import org.specs2.mutable._ +import org.scalatest.{Matchers, WordSpec} -class MutableLRUCacheTest extends Specification { +class MutableLRUCacheTest extends WordSpec with Matchers { def freshCache = MutableLRUCache[String, Int](2) def checkCache(pairs: Seq[(String, Int)], results: Seq[Boolean]) = { val cache = freshCache pairs.foreach(cache += _) - pairs.map { case (k, _) => cache.contains(k) } must be_==(results) + pairs.map { case (k, _) => cache.contains(k) } should equal(results) } "MutableLRUCache works properly with threshold 2" in { diff --git a/storehaus-cache/src/test/scala/com/twitter/storehaus/cache/MutableTTLCacheTest.scala b/storehaus-cache/src/test/scala/com/twitter/storehaus/cache/MutableTTLCacheTest.scala index 385e18c5..3c190965 100644 --- a/storehaus-cache/src/test/scala/com/twitter/storehaus/cache/MutableTTLCacheTest.scala +++ b/storehaus-cache/src/test/scala/com/twitter/storehaus/cache/MutableTTLCacheTest.scala @@ -16,28 +16,28 @@ package com.twitter.storehaus.cache -import org.specs2.mutable._ +import org.scalatest.{Matchers, WordSpec} import com.twitter.util.Duration -class MutableTTLCacheTest extends Specification { +class MutableTTLCacheTest extends WordSpec with Matchers { "TTLCache exhibits proper TTL-ness" in { val ttl: Duration = Duration.fromMilliseconds(500) val cache = MutableCache.ttl[String, Int](ttl, 100) cache += ("a" -> 1) cache += ("b" -> 2) - cache.toNonExpiredMap must be_==(Map("a" -> 1, "b" -> 2)) + cache.toNonExpiredMap should equal(Map("a" -> 1, "b" -> 2)) Thread.sleep(ttl.inMilliseconds) cache += ("c" -> 3) - cache.toNonExpiredMap must be_==(Map("c" -> 3)) + cache.toNonExpiredMap should equal(Map("c" -> 3)) } "TTLCache does not return an expired value" in { val ttl: Duration = Duration.fromMilliseconds(500) val cache = MutableCache.ttl[String, Int](ttl, 100) cache += ("a" -> 10) - cache.get("a") must be_==(Some(10)) + cache.get("a") should equal(Some(10)) Thread.sleep(ttl.inMilliseconds) - cache.get("a") must be_==(None) + cache.get("a") should equal(None) } } diff --git a/storehaus-cache/src/test/scala/com/twitter/storehaus/cache/TTLCacheTest.scala b/storehaus-cache/src/test/scala/com/twitter/storehaus/cache/TTLCacheTest.scala index 1c915dec..9bc66585 100644 --- a/storehaus-cache/src/test/scala/com/twitter/storehaus/cache/TTLCacheTest.scala +++ b/storehaus-cache/src/test/scala/com/twitter/storehaus/cache/TTLCacheTest.scala @@ -16,25 +16,24 @@ package com.twitter.storehaus.cache -import org.specs2.mutable._ +import org.scalatest.{Matchers, WordSpec} import com.twitter.util.Duration - -class TTLCacheTest extends Specification { +class TTLCacheTest extends WordSpec with Matchers { val ttlMS = 600 val cache = Cache.ttl[String, Int](Duration.fromMilliseconds(ttlMS)) "TTLCache exhibits proper TTL-ness" in { val abCache = cache.putClocked("a" -> 1)._2.putClocked("b" -> 2)._2 - abCache.toNonExpiredMap must be_==(Map("a" -> 1, "b" -> 2)) + abCache.toNonExpiredMap should equal(Map("a" -> 1, "b" -> 2)) Thread.sleep(ttlMS) - (abCache.putClocked("c" -> 3)._2).toNonExpiredMap must be_==(Map("c" -> 3)) + (abCache.putClocked("c" -> 3)._2).toNonExpiredMap should equal(Map("c" -> 3)) } "TTLCache does not return an expired value" in { val withV = cache.putClocked("a" -> 10)._2 - withV.getNonExpired("a") must be_==(Some(10)) + withV.getNonExpired("a") should equal(Some(10)) Thread.sleep(ttlMS) - withV.getNonExpired("a") must be_==(None) + withV.getNonExpired("a") should equal(None) } } From cf4d4136fc94fe46a1f4998802886a9f051af75a Mon Sep 17 00:00:00 2001 From: Franklin Hu Date: Mon, 4 May 2015 11:24:55 -1000 Subject: [PATCH 7/9] storehaus-redis: switch to scalatest --- project/Build.scala | 3 +-- .../storehaus/redis/RedisSortedSetSpec.scala | 20 +++++++++---------- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/project/Build.scala b/project/Build.scala index c62d2bb9..073d65db 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -194,8 +194,7 @@ object StorehausBuild extends Build { libraryDependencies ++= Seq ( "com.twitter" %% "bijection-core" % bijectionVersion, "com.twitter" %% "bijection-netty" % bijectionVersion, - "com.twitter" %% "finagle-redis" % finagleVersion, - "org.specs2" %% "specs2" % specs2Version % "test" + "com.twitter" %% "finagle-redis" % finagleVersion ), // we don't want various tests clobbering each others keys parallelExecution in Test := false diff --git a/storehaus-redis/src/test/scala/com/twitter/storehaus/redis/RedisSortedSetSpec.scala b/storehaus-redis/src/test/scala/com/twitter/storehaus/redis/RedisSortedSetSpec.scala index ceb1c47d..050ba889 100644 --- a/storehaus-redis/src/test/scala/com/twitter/storehaus/redis/RedisSortedSetSpec.scala +++ b/storehaus-redis/src/test/scala/com/twitter/storehaus/redis/RedisSortedSetSpec.scala @@ -6,10 +6,10 @@ import com.twitter.storehaus.algebra.MergeableStore import com.twitter.storehaus.testing.CloseableCleanup import com.twitter.util.{ Await, Future } import org.jboss.netty.buffer.ChannelBuffer -import org.specs2.mutable._ +import org.scalatest.{Matchers, WordSpec} import scala.util.Try -class RedisSortedSetSpec extends Specification +class RedisSortedSetSpec extends WordSpec with Matchers with CloseableCleanup[RedisSortedSetStore] with DefaultRedisClient { import com.twitter.bijection.Bijection._ @@ -38,14 +38,12 @@ class RedisSortedSetSpec extends Specification if (xs.isEmpty) None else Some(xs.init, xs.last) } - sequential // Required as tests mutate the store in order - "RedisSortedSet" should { "support Store operations" in { Await.result(for { put <- sets.put(("commits", Some(commits))) commits <- sets.get("commits") - } yield commits) must beSome(commits.sortWith(_._2 < _._2)) + } yield commits) should be(Some(commits.sortWith(_._2 < _._2))) } "support merge operations" in { @@ -53,14 +51,14 @@ class RedisSortedSetSpec extends Specification _ <- sets.merge(("commits", Seq(("sritchie", 1.0)))) commits <- sets.get("commits") } yield commits) - (for (_ ::> last <- merged) yield last) must beSome( - ("sritchie", 138.0)) + (for (_ ::> last <- merged) yield last) should be(Some( + ("sritchie", 138.0))) } "support delete operation" in { Await.result(for { _ <- sets.put(("commits", None)) commits <- sets.get("commits") - } yield commits) must beNone + } yield commits) should be(None) } } @@ -74,7 +72,7 @@ class RedisSortedSetSpec extends Specification Await.result(Future.collect(members.multiPut(putting).values.toSeq)) putting.foreach { case (k, v) => - Await.result(members.get(k)) aka("key %s" format k) must_==(v) + Await.result(members.get(k)) should equal(v) } } "support merge operations" in { @@ -82,14 +80,14 @@ class RedisSortedSetSpec extends Specification Await.result(for { _ <- members.merge((who, 1.0)) score <- members.get(who) - } yield score) aka("score of %s" format who) must beSome(138.0) + } yield score) should be(Some(138.0)) } "support delete operation" in { val who = ("commits", "sritchie") Await.result(for { _ <- members.put((who, None)) score <- members.get(who) - } yield score) aka("score of %s" format who) must beNone + } yield score) should be(None) } } } From d2d45c6e0eff17c20493a380226aa7e5dbaa69fe Mon Sep 17 00:00:00 2001 From: Franklin Hu Date: Mon, 4 May 2015 17:13:53 -1000 Subject: [PATCH 8/9] storehaus-elasticsearch: switch to scalatest --- project/Build.scala | 3 +- .../elasticsearch/DefaultElasticContext.scala | 4 +- .../ElasticSearchStoreSpecs.scala | 53 +++++++++---------- 3 files changed, 28 insertions(+), 32 deletions(-) diff --git a/project/Build.scala b/project/Build.scala index 073d65db..3936f852 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -268,8 +268,7 @@ object StorehausBuild extends Build { "org.elasticsearch" % "elasticsearch" % "0.90.9", "org.json4s" %% "json4s-native" % "3.2.6", "com.google.code.findbugs" % "jsr305" % "1.3.+", - "com.twitter" %% "bijection-json4s" % bijectionVersion, - "org.specs2" %% "specs2" % specs2Version % "test" + "com.twitter" %% "bijection-json4s" % bijectionVersion ), // we don't want various tests clobbering each others keys parallelExecution in Test := false diff --git a/storehaus-elasticsearch/src/test/scala/com/twitter/storehaus/elasticsearch/DefaultElasticContext.scala b/storehaus-elasticsearch/src/test/scala/com/twitter/storehaus/elasticsearch/DefaultElasticContext.scala index eabf89b3..778a35c2 100644 --- a/storehaus-elasticsearch/src/test/scala/com/twitter/storehaus/elasticsearch/DefaultElasticContext.scala +++ b/storehaus-elasticsearch/src/test/scala/com/twitter/storehaus/elasticsearch/DefaultElasticContext.scala @@ -20,15 +20,13 @@ import org.elasticsearch.common.settings.ImmutableSettings import java.util.UUID import java.io.File import org.elasticsearch.node.NodeBuilder._ -import org.specs2.specification.Scope import org.json4s.{native, NoTypeHints} - /** * @author Mansur Ashraf * @since 1/13/14 */ -trait DefaultElasticContext extends Scope { +trait DefaultElasticContext { val tempFile = File.createTempFile("elasticsearchtests", "tmp") val homeDir = new File(tempFile.getParent + "/" + UUID.randomUUID().toString) diff --git a/storehaus-elasticsearch/src/test/scala/com/twitter/storehaus/elasticsearch/ElasticSearchStoreSpecs.scala b/storehaus-elasticsearch/src/test/scala/com/twitter/storehaus/elasticsearch/ElasticSearchStoreSpecs.scala index 18dd8336..9b87f907 100644 --- a/storehaus-elasticsearch/src/test/scala/com/twitter/storehaus/elasticsearch/ElasticSearchStoreSpecs.scala +++ b/storehaus-elasticsearch/src/test/scala/com/twitter/storehaus/elasticsearch/ElasticSearchStoreSpecs.scala @@ -16,7 +16,7 @@ package com.twitter.storehaus.elasticsearch -import org.specs2.mutable.Specification +import org.scalatest.{OneInstancePerTest, Matchers, WordSpec} import com.twitter.util.{Future, Await} import com.twitter.storehaus.FutureOps import org.elasticsearch.action.search.SearchRequestBuilder @@ -28,8 +28,7 @@ import org.json4s.{native, NoTypeHints} * @author Mansur Ashraf * @since 1/13/14 */ -class ElasticSearchStoreSpecs extends Specification { - sequential +class ElasticSearchStoreSpecs extends WordSpec with Matchers with OneInstancePerTest with DefaultElasticContext { private implicit val formats = native.Serialization.formats(NoTypeHints) @@ -37,49 +36,49 @@ class ElasticSearchStoreSpecs extends Specification { "ElasticSearch Store" should { - "Put a value" in new DefaultElasticContext { - private val key = "put_key" + "Put a value" in { + val key = "put_key" store.put((key, Some(person))) blockAndRefreshIndex val result = Await.result(store.get(key)) - result === Some(person) + result should equal(Some(person)) } - "Retrieve a value that doesnt exist" in new DefaultElasticContext { - private val key = "put_key" + "Retrieve a value that doesnt exist" in { + val key = "put_key" store.put((key, Some(person))) blockAndRefreshIndex val result = Await.result(store.get("missing_key")) - result === None + result should equal(None) } - "Update a value" in new DefaultElasticContext { - private val key = "update_key" + "Update a value" in { + val key = "update_key" store.put(key, Some(person)) store.put(key, Some(person.copy(age = 30))) blockAndRefreshIndex val result = Await.result(store.get(key)) - result === Some(person.copy(age = 30)) + result should equal(Some(person.copy(age = 30))) } "Delete a value" in new DefaultElasticContext { - private val key = "delete_key" + val key = "delete_key" store.put(key, Some(person)) store.put(key, None) blockAndRefreshIndex val result = Await.result(store.get(key)) - result === None + result should equal (None) } - "Put multiple values" in new DefaultElasticContext { + "Put multiple values" in { val key = "_put_key" val persons = (1 to 10).map(i => i + key -> Some(person.copy(age = i))).toMap @@ -89,10 +88,10 @@ class ElasticSearchStoreSpecs extends Specification { val response = store.multiGet(persons.keySet) val result = Await.result(FutureOps.mapCollect(response)) - result === persons + result should equal (persons) } - "Retrieve values that do not exist" in new DefaultElasticContext { + "Retrieve values that do not exist" in { val key = "_put_key" val persons = (1 to 10).map(i => i + key -> Some(person.copy(age = i))).toMap @@ -102,10 +101,10 @@ class ElasticSearchStoreSpecs extends Specification { val response = store.multiGet(Set[String]()) val result = Await.result(FutureOps.mapCollect(response)) - result === Map[String,Future[Option[String]]]() + result should equal(Map[String,Future[Option[String]]]()) } - "Update multiple values" in new DefaultElasticContext { + "Update multiple values" in { val key = "_update_key" val persons = (1 to 10).map(i => i + key -> Some(person.copy(age = i))).toMap @@ -117,10 +116,10 @@ class ElasticSearchStoreSpecs extends Specification { val response = store.multiGet(persons_updated.keySet) val result = Await.result(FutureOps.mapCollect(response)) - result === persons_updated + result should equal(persons_updated) } - "Delete multiple values" in new DefaultElasticContext { + "Delete multiple values" in { val key = "_delete_key" val persons = (1 to 10).map(i => i + key -> Some(person.copy(age = i))).toMap @@ -132,10 +131,10 @@ class ElasticSearchStoreSpecs extends Specification { val response = store.multiGet(deleted_persons.keySet) val result = Await.result(FutureOps.mapCollect(response)) - result === deleted_persons + result should equal(deleted_persons) } - "Search for values" in new DefaultElasticContext { + "Search for values" in { val bookStore = ElasticSearchCaseClassStore[Book]("books", "programming", client) val books = Map( @@ -152,8 +151,8 @@ class ElasticSearchStoreSpecs extends Specification { //search for a particular author val request1 = new SearchRequestBuilder(client).setQuery(termQuery("authors", "josh")).request() val response1 = Await.result(bookStore.queryable.get(request1)) - response1 !== None - response1.get.head.name === "Effective Java" + response1 should not equal(None) + response1.get.head.name should equal("Effective Java") //find all the books published after 2001 where author is not Josh Bloch @@ -167,8 +166,8 @@ class ElasticSearchStoreSpecs extends Specification { ).request() val response2 = Await.result(bookStore.queryable.get(request2)) - response2 !== None - response2.get.size === 2 + response2 should not equal(None) + response2.get.size should equal(2) } } From df7f964081bc12e3dc2d7f602c165dce06f1435d Mon Sep 17 00:00:00 2001 From: Joe Nievelt Date: Wed, 13 May 2015 14:31:41 -0700 Subject: [PATCH 9/9] Prepare for 0.11.0 release --- .travis.yml | 2 +- CHANGES.md | 6 ++++++ project/Build.scala | 14 +++++++------- sbt | 4 ++-- 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1429c763..b39494e3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ language: scala sudo: false scala: - - 2.10.4 + - 2.10.5 before_script: - mysql -u root -e "create database storehaus_test;" - mysql -u root -e "create user 'storehaususer'@'localhost' identified by 'test1234';" diff --git a/CHANGES.md b/CHANGES.md index ad1a0e69..017296e0 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,11 @@ # Storehaus # +### Version 0.11.0 ### +* Add correct String/ChannelBuffer injections #257 +* initial scalatest migration #260 +* Remove usage of twitter's maven repo, travis seems to dislike it -- mayb... #261 +* Bijection 0.8.0, algebird 0.10.0, scalding 0.14.0, and scala 2.10.5 + ### Version 0.10.0 ### * Use latest scalding, algebird, and bijection versions: https://github.com/twitter/storehaus/pull/255 * Use new Travis CI container infrastructure: https://github.com/twitter/storehaus/pull/254 diff --git a/project/Build.scala b/project/Build.scala index 3936f852..19914dc4 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -51,9 +51,9 @@ object StorehausBuild extends Build { val sharedSettings = extraSettings ++ ciSettings ++ Seq( organization := "com.twitter", - scalaVersion := "2.10.4", - version := "0.10.0", - crossScalaVersions := Seq("2.10.4"), + scalaVersion := "2.10.5", + version := "0.11.0", + crossScalaVersions := Seq("2.10.5"), javacOptions ++= Seq("-source", "1.6", "-target", "1.6"), javacOptions in doc := Seq("-source", "1.6"), libraryDependencies += "org.scalatest" %% "scalatest" % scalatestVersion % "test", @@ -110,12 +110,12 @@ object StorehausBuild extends Build { def youngestForwardCompatible(subProj: String) = Some(subProj) .filterNot(unreleasedModules.contains(_)) - .map { s => "com.twitter" % ("storehaus-" + s + "_2.10") % "0.10.0" } + .map { s => "com.twitter" % ("storehaus-" + s + "_2.10") % "0.11.0" } - val algebirdVersion = "0.9.0" - val bijectionVersion = "0.7.2" + val algebirdVersion = "0.10.0" + val bijectionVersion = "0.8.0" val utilVersion = "6.22.0" - val scaldingVersion = "0.13.1" + val scaldingVersion = "0.14.0" val finagleVersion = "6.22.0" val scalatestVersion = "2.2.4" val specs2Version = "1.13" diff --git a/sbt b/sbt index 148bbb93..689cb94c 100755 --- a/sbt +++ b/sbt @@ -126,8 +126,8 @@ declare -r default_jvm_opts="-Dfile.encoding=UTF8 -XX:MaxPermSize=384m -Xms512m declare -r noshare_opts="-Dsbt.global.base=project/.sbtboot -Dsbt.boot.directory=project/.boot -Dsbt.ivy.home=project/.ivy" declare -r latest_28="2.8.2" declare -r latest_29="2.9.3" -declare -r latest_210="2.10.3" -declare -r latest_211="2.11.0-M5" +declare -r latest_210="2.10.5" +declare -r latest_211="2.11.5" declare -r script_path=$(get_script_path "$BASH_SOURCE") declare -r script_dir="$(dirname $script_path)"